package weblech.spider;

import java.util.List;
import java.util.ArrayList;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.commons.lang.StringUtils;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.util.SimpleNodeIterator;


public class HTMLParser {

    /** 解析document,得到其中的链接地址list */
    public List<URL> parseLinksInDocument(String textContent, boolean requestLink) {
        return parseContent(textContent, requestLink);
    }

    private List<URL> parseContent(String textContent, boolean requestLink) {
        Parser parser = Parser.createParser(textContent, "GBK");
        
        NodeList nodeList = null;
        try {
            nodeList = parser.extractAllNodesThatMatch(new NodeFilter(){
                /**  */
                private static final long serialVersionUID = 5802449404204473679L;

                public boolean accept(Node node) {
                    if ((node instanceof LinkTag) || (node instanceof ImageTag)) {
                        return true;
                    }
                    
                    return false;
                }
            });
        } catch (ParserException e) {
            e.printStackTrace();
        }
        
        List<URL> urls = new ArrayList<URL>();
        if (nodeList != null) {
            SimpleNodeIterator it = nodeList.elements();
            while (it.hasMoreNodes()) {
                Node node = it.nextNode();
                String url = "";
                if (node instanceof ImageTag) {
                    ImageTag imageTag = (ImageTag) node;
                    url = imageTag.getImageURL();
                } else if (requestLink && node instanceof LinkTag){
                    LinkTag linkTag = (LinkTag) node;
                    url = linkTag.getLink();
                }
                
                if (StringUtils.isNotBlank(url) && isNormalUrl(url)) {
                    try {
                        urls.add(new URL(url));
                    } catch (MalformedURLException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
        
        return urls;
    }
    
    private boolean isNormalUrl(String src) {
        return src.startsWith("http://");
    }
   
}
